{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D2H6YLCPOXvZ",
        "outputId": "dbe09365-d861-4fd3-ed79-d8929fb596de"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.23.1-py3-none-any.whl (5.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.3 MB 4.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 70.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.10.1-py3-none-any.whl (163 kB)\n",
            "\u001b[K     |████████████████████████████████| 163 kB 86.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.9.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.10.1 tokenizers-0.13.1 transformers-4.23.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.6.1-py3-none-any.whl (441 kB)\n",
            "\u001b[K     |████████████████████████████████| 441 kB 4.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.8.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.13.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.13-py37-none-any.whl (115 kB)\n",
            "\u001b[K     |████████████████████████████████| 115 kB 82.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.1)\n",
            "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.5.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.10.1)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 84.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (22.1.0)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.8.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.1.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.8.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 77.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.9.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: urllib3, xxhash, responses, multiprocess, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.6.1 multiprocess-0.70.13 responses-0.18.0 urllib3-1.25.11 xxhash-3.1.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (1.21.6)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.3.5)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2022.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.21.6)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n",
            "Archive:  data_and_models.zip\n",
            "   creating: data_and_models/\n",
            "  inflating: __MACOSX/._data_and_models  \n",
            "  inflating: data_and_models/logistic_model_8.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._logistic_model_8.pkl  \n",
            "  inflating: data_and_models/tfidf_44.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._tfidf_44.pkl  \n",
            "  inflating: data_and_models/tfidf_8.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._tfidf_8.pkl  \n",
            "  inflating: data_and_models/target_corpus.csv  \n",
            "  inflating: __MACOSX/data_and_models/._target_corpus.csv  \n",
            "  inflating: data_and_models/logistic_model_44.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._logistic_model_44.pkl  \n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install datasets\n",
        "!pip install --upgrade --no-cache-dir gdown==4.5.4\n",
        "\n",
        "!gdown 18oZZ4jqRK-uF-Nz6ftRdgNjKix88hrnO\n",
        "!unzip data_and_models.zip && rm data_and_models.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "cfeb8cae1ada47c1aed45b4770cc80ee",
            "2b6b3ee87c7a4fb492d64d8da1e0845c",
            "882495f4154e4cb7ab41da052bcc56cd",
            "612d9949943e428ea7b1b263691261a4",
            "86c2471c24e24765a3a102f3d128d11c",
            "f278b8cd217d408c8b17d476eee0e73e",
            "005d0e44d16a4b8e95d207fce8f2fa9b",
            "12ea031fbddd4dd197ff322f89f60089",
            "fe54405e547a4ebe80f663d294df33e9",
            "b3a09d45a792485c8db365c53ab5dd8d",
            "2520b10f52084ceca5d66dbfcb00d976",
            "1dd8e0c5438d47e69beb4ba0a47d6cab",
            "21f3f43494a0413fbe736fa41c7dfa51",
            "132e8a03c4954f70a1f787e5296fca37",
            "3b43d94ef3c64a25915fee2643d9c8c6",
            "45b2aa984e904e5aa4297347177c7b82",
            "b053ad4929094530aee19358b3e91669",
            "1d099a73be5f46b883c1f9d463b7a424",
            "7048c2161e6a4b12b1f74aecc5847174",
            "8c988c814b3f48dea9de3eaa3255c91f",
            "1919fae16694484b94f4272c52b5d97b",
            "87c405bbac3f48d8b799816335199f12",
            "bb210bf5173e4090a702fc3aa5995e46",
            "a4e6dddfe191432b961b506564cccddd",
            "93ceb2df65bf437d970adc95df8b620f",
            "d61cdb598a94451486c9adfeffaeefd5",
            "7f259ab6c6f84b388ab914e3fb3f3804",
            "85d636083d354d398755f364ae08811e",
            "43a1275ed3a846398c0d2e5dcdb54048",
            "60e808b79ad448028dcf9429208c7018",
            "187c716e512548a697074e5bcce4fc18",
            "fcb499e5829c4c8da7985081ac48b9a3",
            "05517a93533449289149aa6b0e5fe672",
            "e4d1f51a95ba4e43874cc30c677c3387",
            "45651d7475c34e5ea03b89fe35e644ab",
            "89cb305328b14ca2b6b485161fe0d295",
            "3b3d68390f134798a2a9bbc73a1081e7",
            "7af66c5f2e2948138942375ace325b75",
            "e4910017bb94482c907310eb6e133ffd",
            "50cd3b6daa0c4a4588b02740fa4fc463",
            "58ff4e5eec7d43b8b362c60f8535879f",
            "7ff32ea967064f8d88de00e9b23f1384",
            "e274353afc5f42fa92e4eaf3bbda876a",
            "2fa8f82a5c8c413e8d8ea47f48dd47ba",
            "ebc1ad964f8f47d2808a2c92556af0a7",
            "6fc774ec1962478aa9e2490d6e597a2a",
            "dffcb57ad5884a46a9020a3cbf6ce5e5",
            "543677234ab04574b17edbeb293f09aa",
            "84ffa11876364d5791331c0a8b905397",
            "391c778ce2c64f1e92445c1de8084bee",
            "ecf3b77d0099414d93bb05877dc1bc8a",
            "17cafcc34f374c1eb0f1f95f609549b5",
            "21f3343889134db39bb10bbdd26ff6a6",
            "f8d5b4c2d20d4e2dbaad7dafdd391bc2",
            "171fca9859d14cca9f314b1675e7e54c",
            "861a0c463424460888fa7dceba3b67d8",
            "a013e26d3b82443681f197fba9d212c1",
            "52bed226b35641bea40884bc01f3ae2b",
            "298577d50d844021b028a243adc57cde",
            "e0bc4793a7b04c3ab660a37401fa1cce",
            "81e4a44d5ee24f27b1e24ffa6aefb787",
            "7b091adad6c448f0a465992d10a25781",
            "3620c805f5d741cab73c251c9447e8c2",
            "b6f89438b3c946f8bceeda59d5208f4d",
            "bcb9a01f77ae4cd6b0520f4e0648df73",
            "09b71a436f2c495e97708494b6c59ad7"
          ]
        },
        "id": "H-LANn-hUlZh",
        "outputId": "58ca9523-c042-4822-a836-62cd98c23660"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cfeb8cae1ada47c1aed45b4770cc80ee",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/899k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1dd8e0c5438d47e69beb4ba0a47d6cab",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bb210bf5173e4090a702fc3aa5995e46",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e4d1f51a95ba4e43874cc30c677c3387",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/481 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 36\n",
            "# classes in test 35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ebc1ad964f8f47d2808a2c92556af0a7",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/501M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 05:54, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.621700</td>\n",
              "      <td>2.499005</td>\n",
              "      <td>0.385600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.211800</td>\n",
              "      <td>2.175195</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.831200</td>\n",
              "      <td>2.090887</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.769900</td>\n",
              "      <td>2.043772</td>\n",
              "      <td>0.497600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.314400</td>\n",
              "      <td>2.143308</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.191600</td>\n",
              "      <td>2.247154</td>\n",
              "      <td>0.470400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.922200</td>\n",
              "      <td>2.221262</td>\n",
              "      <td>0.480000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.478500</td>\n",
              "      <td>2.344559</td>\n",
              "      <td>0.460800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.544400</td>\n",
              "      <td>2.501603</td>\n",
              "      <td>0.459200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.418500</td>\n",
              "      <td>2.493924</td>\n",
              "      <td>0.465600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.362700</td>\n",
              "      <td>2.587570</td>\n",
              "      <td>0.459200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.294600</td>\n",
              "      <td>2.721775</td>\n",
              "      <td>0.468800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.183100</td>\n",
              "      <td>2.708038</td>\n",
              "      <td>0.473600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.194500</td>\n",
              "      <td>2.766356</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.068300</td>\n",
              "      <td>2.902098</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.128900</td>\n",
              "      <td>2.960262</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.108000</td>\n",
              "      <td>3.084119</td>\n",
              "      <td>0.468800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.088100</td>\n",
              "      <td>3.067957</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.023300</td>\n",
              "      <td>3.087142</td>\n",
              "      <td>0.481600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.024000</td>\n",
              "      <td>3.085043</td>\n",
              "      <td>0.488000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:13: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",
            "  del sys.path[0]\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "861a0c463424460888fa7dceba3b67d8",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/1.65k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 2.0437722206115723).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 38\n",
            "# classes in test 36\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:33, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.578400</td>\n",
              "      <td>2.494752</td>\n",
              "      <td>0.388800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.300900</td>\n",
              "      <td>2.285124</td>\n",
              "      <td>0.419200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.712700</td>\n",
              "      <td>2.137380</td>\n",
              "      <td>0.460800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.522200</td>\n",
              "      <td>2.169816</td>\n",
              "      <td>0.473600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.227500</td>\n",
              "      <td>2.191029</td>\n",
              "      <td>0.457600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.927400</td>\n",
              "      <td>2.270136</td>\n",
              "      <td>0.462400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.888300</td>\n",
              "      <td>2.319786</td>\n",
              "      <td>0.457600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.495800</td>\n",
              "      <td>2.372674</td>\n",
              "      <td>0.460800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.458900</td>\n",
              "      <td>2.494482</td>\n",
              "      <td>0.460800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.382200</td>\n",
              "      <td>2.557843</td>\n",
              "      <td>0.467200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.343500</td>\n",
              "      <td>2.602077</td>\n",
              "      <td>0.452800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.207100</td>\n",
              "      <td>2.661279</td>\n",
              "      <td>0.452800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.213200</td>\n",
              "      <td>2.747055</td>\n",
              "      <td>0.459200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.145500</td>\n",
              "      <td>2.870366</td>\n",
              "      <td>0.448000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.199700</td>\n",
              "      <td>2.954898</td>\n",
              "      <td>0.457600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.111100</td>\n",
              "      <td>3.042625</td>\n",
              "      <td>0.449600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.063700</td>\n",
              "      <td>3.091025</td>\n",
              "      <td>0.451200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.107300</td>\n",
              "      <td>3.152532</td>\n",
              "      <td>0.449600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.043600</td>\n",
              "      <td>3.174242</td>\n",
              "      <td>0.451200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.052200</td>\n",
              "      <td>3.180133</td>\n",
              "      <td>0.454400</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 2.137380361557007).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 41\n",
            "# classes in dev 38\n",
            "# classes in test 37\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:40, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.577600</td>\n",
              "      <td>2.530531</td>\n",
              "      <td>0.379200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.176300</td>\n",
              "      <td>2.382901</td>\n",
              "      <td>0.433600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.907900</td>\n",
              "      <td>2.250292</td>\n",
              "      <td>0.456000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.584300</td>\n",
              "      <td>2.276685</td>\n",
              "      <td>0.440000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.238700</td>\n",
              "      <td>2.399720</td>\n",
              "      <td>0.444800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.956200</td>\n",
              "      <td>2.480031</td>\n",
              "      <td>0.419200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.762500</td>\n",
              "      <td>2.538512</td>\n",
              "      <td>0.425600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.627700</td>\n",
              "      <td>2.471473</td>\n",
              "      <td>0.464000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.625900</td>\n",
              "      <td>2.677087</td>\n",
              "      <td>0.451200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.426300</td>\n",
              "      <td>2.737224</td>\n",
              "      <td>0.451200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.317800</td>\n",
              "      <td>2.854079</td>\n",
              "      <td>0.456000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.264300</td>\n",
              "      <td>2.854780</td>\n",
              "      <td>0.448000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.125200</td>\n",
              "      <td>2.990844</td>\n",
              "      <td>0.441600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.205700</td>\n",
              "      <td>3.095444</td>\n",
              "      <td>0.433600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.104300</td>\n",
              "      <td>3.240748</td>\n",
              "      <td>0.441600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.086400</td>\n",
              "      <td>3.243804</td>\n",
              "      <td>0.444800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.059200</td>\n",
              "      <td>3.289966</td>\n",
              "      <td>0.454400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.045300</td>\n",
              "      <td>3.354967</td>\n",
              "      <td>0.441600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.048700</td>\n",
              "      <td>3.390336</td>\n",
              "      <td>0.446400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.023500</td>\n",
              "      <td>3.400526</td>\n",
              "      <td>0.454400</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 2.2502923011779785).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 35\n",
            "# classes in test 37\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:36, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.665400</td>\n",
              "      <td>2.620600</td>\n",
              "      <td>0.345600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.340100</td>\n",
              "      <td>2.360425</td>\n",
              "      <td>0.408000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.832000</td>\n",
              "      <td>2.271099</td>\n",
              "      <td>0.411200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.406100</td>\n",
              "      <td>2.325723</td>\n",
              "      <td>0.435200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.239100</td>\n",
              "      <td>2.369185</td>\n",
              "      <td>0.417600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>1.102200</td>\n",
              "      <td>2.463613</td>\n",
              "      <td>0.409600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.913400</td>\n",
              "      <td>2.611687</td>\n",
              "      <td>0.396800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.453200</td>\n",
              "      <td>2.594323</td>\n",
              "      <td>0.408000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.483300</td>\n",
              "      <td>2.640660</td>\n",
              "      <td>0.406400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.393800</td>\n",
              "      <td>2.798657</td>\n",
              "      <td>0.398400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.236000</td>\n",
              "      <td>2.864151</td>\n",
              "      <td>0.425600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.325200</td>\n",
              "      <td>2.961178</td>\n",
              "      <td>0.414400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.224600</td>\n",
              "      <td>3.098228</td>\n",
              "      <td>0.393600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.216800</td>\n",
              "      <td>3.099358</td>\n",
              "      <td>0.406400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.140600</td>\n",
              "      <td>3.146765</td>\n",
              "      <td>0.408000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.176600</td>\n",
              "      <td>3.275793</td>\n",
              "      <td>0.408000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.056600</td>\n",
              "      <td>3.320166</td>\n",
              "      <td>0.411200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.048700</td>\n",
              "      <td>3.415085</td>\n",
              "      <td>0.392000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.044600</td>\n",
              "      <td>3.431285</td>\n",
              "      <td>0.408000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.053400</td>\n",
              "      <td>3.449994</td>\n",
              "      <td>0.400000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 2.271099328994751).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 38\n",
            "# classes in test 35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:27, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.769500</td>\n",
              "      <td>2.648046</td>\n",
              "      <td>0.361600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.173100</td>\n",
              "      <td>2.356622</td>\n",
              "      <td>0.424000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.970500</td>\n",
              "      <td>2.283024</td>\n",
              "      <td>0.430400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.599400</td>\n",
              "      <td>2.235606</td>\n",
              "      <td>0.438400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.428800</td>\n",
              "      <td>2.392470</td>\n",
              "      <td>0.425600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.990900</td>\n",
              "      <td>2.399296</td>\n",
              "      <td>0.430400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.860000</td>\n",
              "      <td>2.498942</td>\n",
              "      <td>0.441600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.566400</td>\n",
              "      <td>2.595352</td>\n",
              "      <td>0.428800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.434900</td>\n",
              "      <td>2.710012</td>\n",
              "      <td>0.443200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.479500</td>\n",
              "      <td>2.841173</td>\n",
              "      <td>0.419200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.239000</td>\n",
              "      <td>2.851869</td>\n",
              "      <td>0.422400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.282900</td>\n",
              "      <td>3.006221</td>\n",
              "      <td>0.417600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.190900</td>\n",
              "      <td>3.094927</td>\n",
              "      <td>0.425600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.141700</td>\n",
              "      <td>3.160009</td>\n",
              "      <td>0.420800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.072900</td>\n",
              "      <td>3.275676</td>\n",
              "      <td>0.424000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.076300</td>\n",
              "      <td>3.365600</td>\n",
              "      <td>0.433600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.072700</td>\n",
              "      <td>3.435350</td>\n",
              "      <td>0.432000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.033400</td>\n",
              "      <td>3.508399</td>\n",
              "      <td>0.427200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.061900</td>\n",
              "      <td>3.519186</td>\n",
              "      <td>0.428800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.043000</td>\n",
              "      <td>3.542867</td>\n",
              "      <td>0.428800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 2.2356059551239014).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 36\n",
            "# classes in test 35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:13, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.490700</td>\n",
              "      <td>2.291392</td>\n",
              "      <td>0.443200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.003200</td>\n",
              "      <td>1.939445</td>\n",
              "      <td>0.521600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.622700</td>\n",
              "      <td>1.810220</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.358000</td>\n",
              "      <td>1.819359</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.028200</td>\n",
              "      <td>1.844699</td>\n",
              "      <td>0.524800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.864200</td>\n",
              "      <td>1.910618</td>\n",
              "      <td>0.526400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.600300</td>\n",
              "      <td>1.951114</td>\n",
              "      <td>0.537600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.395600</td>\n",
              "      <td>2.024622</td>\n",
              "      <td>0.524800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.379100</td>\n",
              "      <td>2.067181</td>\n",
              "      <td>0.529600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.251800</td>\n",
              "      <td>2.118134</td>\n",
              "      <td>0.516800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.249400</td>\n",
              "      <td>2.213807</td>\n",
              "      <td>0.536000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.124500</td>\n",
              "      <td>2.251400</td>\n",
              "      <td>0.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.088600</td>\n",
              "      <td>2.355387</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.165800</td>\n",
              "      <td>2.563812</td>\n",
              "      <td>0.516800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.045700</td>\n",
              "      <td>2.557687</td>\n",
              "      <td>0.526400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.034900</td>\n",
              "      <td>2.642665</td>\n",
              "      <td>0.536000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.046800</td>\n",
              "      <td>2.611371</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.065900</td>\n",
              "      <td>2.695418</td>\n",
              "      <td>0.537600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.012300</td>\n",
              "      <td>2.691458</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.010600</td>\n",
              "      <td>2.674776</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.8102198839187622).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 38\n",
            "# classes in test 36\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:18, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.513600</td>\n",
              "      <td>2.331985</td>\n",
              "      <td>0.425600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.176200</td>\n",
              "      <td>2.072746</td>\n",
              "      <td>0.468800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.549900</td>\n",
              "      <td>1.929446</td>\n",
              "      <td>0.502400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.379900</td>\n",
              "      <td>1.832429</td>\n",
              "      <td>0.518400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.966200</td>\n",
              "      <td>1.879472</td>\n",
              "      <td>0.521600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.754100</td>\n",
              "      <td>1.921443</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.729300</td>\n",
              "      <td>1.994064</td>\n",
              "      <td>0.529600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.295100</td>\n",
              "      <td>2.085191</td>\n",
              "      <td>0.526400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.400600</td>\n",
              "      <td>2.184536</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.252600</td>\n",
              "      <td>2.320652</td>\n",
              "      <td>0.528000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.210500</td>\n",
              "      <td>2.367908</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.123900</td>\n",
              "      <td>2.437009</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.117000</td>\n",
              "      <td>2.496348</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.065500</td>\n",
              "      <td>2.561485</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.088000</td>\n",
              "      <td>2.656824</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.052800</td>\n",
              "      <td>2.695706</td>\n",
              "      <td>0.537600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.021300</td>\n",
              "      <td>2.758168</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.038600</td>\n",
              "      <td>2.818071</td>\n",
              "      <td>0.536000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.055900</td>\n",
              "      <td>2.812459</td>\n",
              "      <td>0.529600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.013200</td>\n",
              "      <td>2.813767</td>\n",
              "      <td>0.529600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 1.8324286937713623).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 41\n",
            "# classes in dev 38\n",
            "# classes in test 37\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:16, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.539100</td>\n",
              "      <td>2.442890</td>\n",
              "      <td>0.414400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.991000</td>\n",
              "      <td>2.177721</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.693600</td>\n",
              "      <td>2.007495</td>\n",
              "      <td>0.494400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.399500</td>\n",
              "      <td>2.003397</td>\n",
              "      <td>0.502400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.074800</td>\n",
              "      <td>2.062061</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.827200</td>\n",
              "      <td>2.090632</td>\n",
              "      <td>0.515200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.652700</td>\n",
              "      <td>2.137616</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.467200</td>\n",
              "      <td>2.197981</td>\n",
              "      <td>0.505600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.545900</td>\n",
              "      <td>2.278202</td>\n",
              "      <td>0.500800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.308300</td>\n",
              "      <td>2.385895</td>\n",
              "      <td>0.502400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.199900</td>\n",
              "      <td>2.504985</td>\n",
              "      <td>0.516800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.220000</td>\n",
              "      <td>2.635213</td>\n",
              "      <td>0.497600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.103000</td>\n",
              "      <td>2.759393</td>\n",
              "      <td>0.481600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.129100</td>\n",
              "      <td>2.920424</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.077800</td>\n",
              "      <td>2.963534</td>\n",
              "      <td>0.504000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.037500</td>\n",
              "      <td>3.011091</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.028600</td>\n",
              "      <td>3.055299</td>\n",
              "      <td>0.502400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.028000</td>\n",
              "      <td>3.072106</td>\n",
              "      <td>0.507200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.028200</td>\n",
              "      <td>3.134322</td>\n",
              "      <td>0.497600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.011100</td>\n",
              "      <td>3.131207</td>\n",
              "      <td>0.505600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 2.003396987915039).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 35\n",
            "# classes in test 37\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:05, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.476900</td>\n",
              "      <td>2.479219</td>\n",
              "      <td>0.380800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.045400</td>\n",
              "      <td>2.143213</td>\n",
              "      <td>0.456000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.536100</td>\n",
              "      <td>2.043451</td>\n",
              "      <td>0.480000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.362000</td>\n",
              "      <td>1.986266</td>\n",
              "      <td>0.468800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.951800</td>\n",
              "      <td>1.988320</td>\n",
              "      <td>0.494400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.808000</td>\n",
              "      <td>2.087839</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.675900</td>\n",
              "      <td>2.203150</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.315300</td>\n",
              "      <td>2.291127</td>\n",
              "      <td>0.459200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.379600</td>\n",
              "      <td>2.288975</td>\n",
              "      <td>0.481600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.259400</td>\n",
              "      <td>2.431520</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.215000</td>\n",
              "      <td>2.528511</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.185500</td>\n",
              "      <td>2.629592</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.183500</td>\n",
              "      <td>2.673608</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.093800</td>\n",
              "      <td>2.723088</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.064400</td>\n",
              "      <td>2.821197</td>\n",
              "      <td>0.484800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.075000</td>\n",
              "      <td>2.946783</td>\n",
              "      <td>0.497600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.014100</td>\n",
              "      <td>2.975064</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.027000</td>\n",
              "      <td>3.026869</td>\n",
              "      <td>0.481600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.013000</td>\n",
              "      <td>3.026489</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.028700</td>\n",
              "      <td>3.052438</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 1.9862662553787231).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 38\n",
            "# classes in test 35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:15, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.605200</td>\n",
              "      <td>2.428265</td>\n",
              "      <td>0.422400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.946600</td>\n",
              "      <td>2.156282</td>\n",
              "      <td>0.460800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.771200</td>\n",
              "      <td>2.005743</td>\n",
              "      <td>0.480000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.174200</td>\n",
              "      <td>2.016110</td>\n",
              "      <td>0.460800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.059200</td>\n",
              "      <td>2.153371</td>\n",
              "      <td>0.459200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.697400</td>\n",
              "      <td>2.158676</td>\n",
              "      <td>0.481600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.589700</td>\n",
              "      <td>2.246525</td>\n",
              "      <td>0.481600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.376000</td>\n",
              "      <td>2.383993</td>\n",
              "      <td>0.465600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.274400</td>\n",
              "      <td>2.449046</td>\n",
              "      <td>0.467200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.242400</td>\n",
              "      <td>2.485710</td>\n",
              "      <td>0.476800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.153400</td>\n",
              "      <td>2.582793</td>\n",
              "      <td>0.465600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.216300</td>\n",
              "      <td>2.756227</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.067700</td>\n",
              "      <td>2.922783</td>\n",
              "      <td>0.464000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.059100</td>\n",
              "      <td>3.050058</td>\n",
              "      <td>0.464000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.023200</td>\n",
              "      <td>3.103093</td>\n",
              "      <td>0.478400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.065500</td>\n",
              "      <td>3.193799</td>\n",
              "      <td>0.470400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.031300</td>\n",
              "      <td>3.216562</td>\n",
              "      <td>0.473600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.030600</td>\n",
              "      <td>3.277661</td>\n",
              "      <td>0.470400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.034200</td>\n",
              "      <td>3.296395</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.021400</td>\n",
              "      <td>3.304898</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 2.0057432651519775).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 36\n",
            "# classes in test 35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:22, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.430300</td>\n",
              "      <td>2.288778</td>\n",
              "      <td>0.417600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.917100</td>\n",
              "      <td>1.934722</td>\n",
              "      <td>0.516800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.582500</td>\n",
              "      <td>1.823273</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.362400</td>\n",
              "      <td>1.823743</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.972300</td>\n",
              "      <td>1.889241</td>\n",
              "      <td>0.523200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.848000</td>\n",
              "      <td>1.888951</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.573900</td>\n",
              "      <td>2.010903</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.409200</td>\n",
              "      <td>1.989290</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.317400</td>\n",
              "      <td>2.029555</td>\n",
              "      <td>0.563200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.230900</td>\n",
              "      <td>2.128701</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.196900</td>\n",
              "      <td>2.317828</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.136000</td>\n",
              "      <td>2.398464</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.049600</td>\n",
              "      <td>2.501200</td>\n",
              "      <td>0.545600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.136700</td>\n",
              "      <td>2.545634</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.057400</td>\n",
              "      <td>2.664694</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.082100</td>\n",
              "      <td>2.783216</td>\n",
              "      <td>0.531200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.014700</td>\n",
              "      <td>2.788378</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.063300</td>\n",
              "      <td>2.824948</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.027000</td>\n",
              "      <td>2.827253</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.009300</td>\n",
              "      <td>2.841008</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.8232734203338623).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 38\n",
            "# classes in test 36\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:12, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.440100</td>\n",
              "      <td>2.280676</td>\n",
              "      <td>0.435200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.108400</td>\n",
              "      <td>2.000825</td>\n",
              "      <td>0.473600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.461600</td>\n",
              "      <td>1.840411</td>\n",
              "      <td>0.512000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.201400</td>\n",
              "      <td>1.784630</td>\n",
              "      <td>0.528000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.943700</td>\n",
              "      <td>1.797061</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.615400</td>\n",
              "      <td>1.897894</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.641900</td>\n",
              "      <td>1.956028</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.229300</td>\n",
              "      <td>2.036530</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.324900</td>\n",
              "      <td>2.070697</td>\n",
              "      <td>0.537600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.199200</td>\n",
              "      <td>2.198223</td>\n",
              "      <td>0.531200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.191800</td>\n",
              "      <td>2.220739</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.091500</td>\n",
              "      <td>2.420457</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.148100</td>\n",
              "      <td>2.490360</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.057200</td>\n",
              "      <td>2.602995</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.028600</td>\n",
              "      <td>2.631632</td>\n",
              "      <td>0.545600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.048200</td>\n",
              "      <td>2.713568</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.013300</td>\n",
              "      <td>2.808079</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.036600</td>\n",
              "      <td>2.792651</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.038100</td>\n",
              "      <td>2.815949</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.008900</td>\n",
              "      <td>2.823409</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 1.7846299409866333).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 41\n",
            "# classes in dev 38\n",
            "# classes in test 37\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:21, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.507300</td>\n",
              "      <td>2.389546</td>\n",
              "      <td>0.403200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.970200</td>\n",
              "      <td>2.135563</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.704300</td>\n",
              "      <td>1.997115</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.453000</td>\n",
              "      <td>2.035100</td>\n",
              "      <td>0.480000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.956900</td>\n",
              "      <td>2.062247</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.694500</td>\n",
              "      <td>2.155730</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.592900</td>\n",
              "      <td>2.221344</td>\n",
              "      <td>0.508800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.499300</td>\n",
              "      <td>2.261892</td>\n",
              "      <td>0.494400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.432800</td>\n",
              "      <td>2.324211</td>\n",
              "      <td>0.505600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.256400</td>\n",
              "      <td>2.473519</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.162800</td>\n",
              "      <td>2.528510</td>\n",
              "      <td>0.510400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.150600</td>\n",
              "      <td>2.699227</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.060500</td>\n",
              "      <td>2.807323</td>\n",
              "      <td>0.499200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.099500</td>\n",
              "      <td>2.924183</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.064200</td>\n",
              "      <td>3.009894</td>\n",
              "      <td>0.499200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.028200</td>\n",
              "      <td>3.092520</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.066600</td>\n",
              "      <td>3.182685</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.041900</td>\n",
              "      <td>3.218215</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.058300</td>\n",
              "      <td>3.211842</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.010000</td>\n",
              "      <td>3.221166</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.9971150159835815).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 35\n",
            "# classes in test 37\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:14, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.514900</td>\n",
              "      <td>2.500490</td>\n",
              "      <td>0.364800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.062200</td>\n",
              "      <td>2.101844</td>\n",
              "      <td>0.464000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.538500</td>\n",
              "      <td>1.987228</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.217100</td>\n",
              "      <td>1.942860</td>\n",
              "      <td>0.502400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.870300</td>\n",
              "      <td>1.969932</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.730900</td>\n",
              "      <td>2.034905</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.683200</td>\n",
              "      <td>2.094531</td>\n",
              "      <td>0.499200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.339300</td>\n",
              "      <td>2.186576</td>\n",
              "      <td>0.488000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.295700</td>\n",
              "      <td>2.145506</td>\n",
              "      <td>0.518400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.235400</td>\n",
              "      <td>2.275206</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.127400</td>\n",
              "      <td>2.396797</td>\n",
              "      <td>0.505600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.302800</td>\n",
              "      <td>2.490853</td>\n",
              "      <td>0.500800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.092300</td>\n",
              "      <td>2.565019</td>\n",
              "      <td>0.513600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.062900</td>\n",
              "      <td>2.657266</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.052800</td>\n",
              "      <td>2.758803</td>\n",
              "      <td>0.523200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.066600</td>\n",
              "      <td>2.822627</td>\n",
              "      <td>0.513600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.041400</td>\n",
              "      <td>2.914744</td>\n",
              "      <td>0.513600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.017700</td>\n",
              "      <td>2.901634</td>\n",
              "      <td>0.521600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.020900</td>\n",
              "      <td>2.928650</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.022300</td>\n",
              "      <td>2.942898</td>\n",
              "      <td>0.513600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 1.9428601264953613).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 38\n",
            "# classes in test 35\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:08, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.592000</td>\n",
              "      <td>2.406589</td>\n",
              "      <td>0.424000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.819100</td>\n",
              "      <td>2.115860</td>\n",
              "      <td>0.462400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.789500</td>\n",
              "      <td>1.973715</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.187000</td>\n",
              "      <td>1.987817</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.018000</td>\n",
              "      <td>2.026030</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.683700</td>\n",
              "      <td>2.087702</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.511400</td>\n",
              "      <td>2.160427</td>\n",
              "      <td>0.484800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.400500</td>\n",
              "      <td>2.265558</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.275900</td>\n",
              "      <td>2.363732</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.168700</td>\n",
              "      <td>2.525493</td>\n",
              "      <td>0.480000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.140500</td>\n",
              "      <td>2.654514</td>\n",
              "      <td>0.478400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.126400</td>\n",
              "      <td>2.768090</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.065700</td>\n",
              "      <td>2.897894</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.060300</td>\n",
              "      <td>2.967344</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.018900</td>\n",
              "      <td>3.023528</td>\n",
              "      <td>0.478400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.038000</td>\n",
              "      <td>3.081142</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.036500</td>\n",
              "      <td>3.214539</td>\n",
              "      <td>0.470400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.013900</td>\n",
              "      <td>3.237193</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.035000</td>\n",
              "      <td>3.215718</td>\n",
              "      <td>0.473600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.034400</td>\n",
              "      <td>3.229725</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.9737151861190796).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:14, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.378900</td>\n",
              "      <td>1.337807</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.098800</td>\n",
              "      <td>1.342990</td>\n",
              "      <td>0.545600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.979400</td>\n",
              "      <td>1.298765</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.968600</td>\n",
              "      <td>1.396530</td>\n",
              "      <td>0.560000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.631100</td>\n",
              "      <td>1.525748</td>\n",
              "      <td>0.556800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.221800</td>\n",
              "      <td>1.831988</td>\n",
              "      <td>0.555200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.257100</td>\n",
              "      <td>1.997571</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.084600</td>\n",
              "      <td>2.182239</td>\n",
              "      <td>0.571200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.086000</td>\n",
              "      <td>2.467403</td>\n",
              "      <td>0.555200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.076300</td>\n",
              "      <td>2.599860</td>\n",
              "      <td>0.561600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.061100</td>\n",
              "      <td>2.868308</td>\n",
              "      <td>0.555200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.049000</td>\n",
              "      <td>3.121061</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.019800</td>\n",
              "      <td>3.192683</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.003300</td>\n",
              "      <td>3.332852</td>\n",
              "      <td>0.550400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>3.376710</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.002200</td>\n",
              "      <td>3.467619</td>\n",
              "      <td>0.555200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.008300</td>\n",
              "      <td>3.509374</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.546050</td>\n",
              "      <td>0.537600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.549980</td>\n",
              "      <td>0.545600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.563285</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.298764705657959).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:21, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.316400</td>\n",
              "      <td>1.388045</td>\n",
              "      <td>0.529600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.131300</td>\n",
              "      <td>1.283970</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.875900</td>\n",
              "      <td>1.368154</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.655600</td>\n",
              "      <td>1.457107</td>\n",
              "      <td>0.566400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.472400</td>\n",
              "      <td>1.533135</td>\n",
              "      <td>0.571200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.301000</td>\n",
              "      <td>1.711547</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.212300</td>\n",
              "      <td>1.930853</td>\n",
              "      <td>0.555200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.163500</td>\n",
              "      <td>2.252084</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.033600</td>\n",
              "      <td>2.408660</td>\n",
              "      <td>0.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.053300</td>\n",
              "      <td>2.631430</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.012500</td>\n",
              "      <td>2.998785</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.013400</td>\n",
              "      <td>3.054894</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.003100</td>\n",
              "      <td>3.174664</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.002100</td>\n",
              "      <td>3.333942</td>\n",
              "      <td>0.556800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.039700</td>\n",
              "      <td>3.387478</td>\n",
              "      <td>0.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>3.440438</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.497202</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.002200</td>\n",
              "      <td>3.477692</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.047400</td>\n",
              "      <td>3.481204</td>\n",
              "      <td>0.561600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.001100</td>\n",
              "      <td>3.459577</td>\n",
              "      <td>0.566400</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.2839704751968384).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:23, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.376200</td>\n",
              "      <td>1.393996</td>\n",
              "      <td>0.523200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.113800</td>\n",
              "      <td>1.325008</td>\n",
              "      <td>0.555200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.844700</td>\n",
              "      <td>1.406461</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.714000</td>\n",
              "      <td>1.473362</td>\n",
              "      <td>0.576000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.466700</td>\n",
              "      <td>1.633926</td>\n",
              "      <td>0.556800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.416500</td>\n",
              "      <td>1.770823</td>\n",
              "      <td>0.550400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.193300</td>\n",
              "      <td>1.948338</td>\n",
              "      <td>0.550400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.197500</td>\n",
              "      <td>2.053538</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.162000</td>\n",
              "      <td>2.501997</td>\n",
              "      <td>0.545600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.082300</td>\n",
              "      <td>2.675789</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.038400</td>\n",
              "      <td>2.887852</td>\n",
              "      <td>0.550400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.015300</td>\n",
              "      <td>3.133565</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.066000</td>\n",
              "      <td>3.231283</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.051100</td>\n",
              "      <td>3.400716</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.002600</td>\n",
              "      <td>3.342756</td>\n",
              "      <td>0.555200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.031900</td>\n",
              "      <td>3.464495</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.009000</td>\n",
              "      <td>3.544098</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.001000</td>\n",
              "      <td>3.534306</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>3.558129</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>3.622802</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.325007677078247).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:09, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.482400</td>\n",
              "      <td>1.424911</td>\n",
              "      <td>0.494400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.311100</td>\n",
              "      <td>1.317304</td>\n",
              "      <td>0.528000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.921000</td>\n",
              "      <td>1.312024</td>\n",
              "      <td>0.561600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.643300</td>\n",
              "      <td>1.475935</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.670200</td>\n",
              "      <td>1.528984</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.396400</td>\n",
              "      <td>1.603358</td>\n",
              "      <td>0.568000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.293300</td>\n",
              "      <td>1.971731</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.146600</td>\n",
              "      <td>2.129893</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.103700</td>\n",
              "      <td>2.377722</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.136700</td>\n",
              "      <td>2.629594</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.033400</td>\n",
              "      <td>2.764947</td>\n",
              "      <td>0.531200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.005900</td>\n",
              "      <td>3.127800</td>\n",
              "      <td>0.521600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.035300</td>\n",
              "      <td>3.152548</td>\n",
              "      <td>0.526400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.023700</td>\n",
              "      <td>3.136283</td>\n",
              "      <td>0.545600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.001100</td>\n",
              "      <td>3.277136</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.001000</td>\n",
              "      <td>3.322598</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.001600</td>\n",
              "      <td>3.375398</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.019500</td>\n",
              "      <td>3.439297</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.022900</td>\n",
              "      <td>3.455782</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>3.469355</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.3120238780975342).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 06:39, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.377200</td>\n",
              "      <td>1.363505</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.159600</td>\n",
              "      <td>1.278537</td>\n",
              "      <td>0.569600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.905400</td>\n",
              "      <td>1.246145</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.665600</td>\n",
              "      <td>1.369946</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.415900</td>\n",
              "      <td>1.643938</td>\n",
              "      <td>0.560000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.365700</td>\n",
              "      <td>1.830832</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.351000</td>\n",
              "      <td>2.120942</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.100700</td>\n",
              "      <td>2.280218</td>\n",
              "      <td>0.582400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.119300</td>\n",
              "      <td>2.666468</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.107000</td>\n",
              "      <td>2.763258</td>\n",
              "      <td>0.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.047900</td>\n",
              "      <td>2.959042</td>\n",
              "      <td>0.560000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.002600</td>\n",
              "      <td>3.059182</td>\n",
              "      <td>0.560000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.010600</td>\n",
              "      <td>3.145193</td>\n",
              "      <td>0.550400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.008100</td>\n",
              "      <td>3.242210</td>\n",
              "      <td>0.572800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.000800</td>\n",
              "      <td>3.300792</td>\n",
              "      <td>0.563200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>3.399088</td>\n",
              "      <td>0.560000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.013000</td>\n",
              "      <td>3.396247</td>\n",
              "      <td>0.561600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.423464</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.427161</td>\n",
              "      <td>0.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.445364</td>\n",
              "      <td>0.560000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.246145248413086).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:12, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.276400</td>\n",
              "      <td>1.212613</td>\n",
              "      <td>0.580800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.979800</td>\n",
              "      <td>1.117378</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.844600</td>\n",
              "      <td>1.141515</td>\n",
              "      <td>0.635200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.914500</td>\n",
              "      <td>1.255076</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.525200</td>\n",
              "      <td>1.370795</td>\n",
              "      <td>0.636800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.203100</td>\n",
              "      <td>1.474914</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.134700</td>\n",
              "      <td>1.787149</td>\n",
              "      <td>0.640000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.147400</td>\n",
              "      <td>1.900893</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.016100</td>\n",
              "      <td>2.206744</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.103200</td>\n",
              "      <td>2.280209</td>\n",
              "      <td>0.643200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.069600</td>\n",
              "      <td>2.410991</td>\n",
              "      <td>0.643200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.028100</td>\n",
              "      <td>2.408396</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.034500</td>\n",
              "      <td>2.526290</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001100</td>\n",
              "      <td>2.582414</td>\n",
              "      <td>0.648000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.608084</td>\n",
              "      <td>0.646400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.608772</td>\n",
              "      <td>0.649600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.650454</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.676279</td>\n",
              "      <td>0.641600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.681459</td>\n",
              "      <td>0.643200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.675398</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.1173778772354126).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:12, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.135200</td>\n",
              "      <td>1.183312</td>\n",
              "      <td>0.576000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.897500</td>\n",
              "      <td>1.110124</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.685100</td>\n",
              "      <td>1.113878</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.620600</td>\n",
              "      <td>1.239940</td>\n",
              "      <td>0.617600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.361300</td>\n",
              "      <td>1.375911</td>\n",
              "      <td>0.622400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.161800</td>\n",
              "      <td>1.470408</td>\n",
              "      <td>0.643200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.210200</td>\n",
              "      <td>1.676568</td>\n",
              "      <td>0.646400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.155400</td>\n",
              "      <td>1.977858</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.041000</td>\n",
              "      <td>2.146773</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.034400</td>\n",
              "      <td>2.358540</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.088100</td>\n",
              "      <td>2.441550</td>\n",
              "      <td>0.643200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.002400</td>\n",
              "      <td>2.507025</td>\n",
              "      <td>0.636800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.037700</td>\n",
              "      <td>2.667887</td>\n",
              "      <td>0.628800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.037600</td>\n",
              "      <td>2.632105</td>\n",
              "      <td>0.643200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.001100</td>\n",
              "      <td>2.720380</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.002900</td>\n",
              "      <td>2.727187</td>\n",
              "      <td>0.636800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.761109</td>\n",
              "      <td>0.641600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.002000</td>\n",
              "      <td>2.792109</td>\n",
              "      <td>0.641600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.797650</td>\n",
              "      <td>0.644800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000400</td>\n",
              "      <td>2.808742</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.110123872756958).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:21, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.236200</td>\n",
              "      <td>1.338504</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.972600</td>\n",
              "      <td>1.212167</td>\n",
              "      <td>0.582400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.746100</td>\n",
              "      <td>1.247118</td>\n",
              "      <td>0.593600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.645600</td>\n",
              "      <td>1.407349</td>\n",
              "      <td>0.601600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.268600</td>\n",
              "      <td>1.621150</td>\n",
              "      <td>0.577600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.298400</td>\n",
              "      <td>1.663277</td>\n",
              "      <td>0.595200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.150400</td>\n",
              "      <td>2.026405</td>\n",
              "      <td>0.587200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.039500</td>\n",
              "      <td>2.343272</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.136500</td>\n",
              "      <td>2.670512</td>\n",
              "      <td>0.561600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.007700</td>\n",
              "      <td>2.748670</td>\n",
              "      <td>0.566400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.013600</td>\n",
              "      <td>2.891477</td>\n",
              "      <td>0.587200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.046600</td>\n",
              "      <td>3.097823</td>\n",
              "      <td>0.579200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.029900</td>\n",
              "      <td>2.974336</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001100</td>\n",
              "      <td>3.073482</td>\n",
              "      <td>0.595200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>3.179734</td>\n",
              "      <td>0.587200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.007000</td>\n",
              "      <td>3.194242</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.144041</td>\n",
              "      <td>0.601600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>3.201563</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.190279</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.192653</td>\n",
              "      <td>0.603200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.2121671438217163).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:29, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.351600</td>\n",
              "      <td>1.308036</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.060200</td>\n",
              "      <td>1.206354</td>\n",
              "      <td>0.592000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.697800</td>\n",
              "      <td>1.266860</td>\n",
              "      <td>0.576000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.522700</td>\n",
              "      <td>1.303626</td>\n",
              "      <td>0.611200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.464200</td>\n",
              "      <td>1.486468</td>\n",
              "      <td>0.585600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.208000</td>\n",
              "      <td>1.646750</td>\n",
              "      <td>0.593600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.190900</td>\n",
              "      <td>1.791024</td>\n",
              "      <td>0.576000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.203500</td>\n",
              "      <td>2.062284</td>\n",
              "      <td>0.566400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.058000</td>\n",
              "      <td>2.314323</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.167000</td>\n",
              "      <td>2.583298</td>\n",
              "      <td>0.587200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.046900</td>\n",
              "      <td>2.600135</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.011800</td>\n",
              "      <td>2.757066</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.002000</td>\n",
              "      <td>2.832330</td>\n",
              "      <td>0.593600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001200</td>\n",
              "      <td>2.874299</td>\n",
              "      <td>0.593600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.001100</td>\n",
              "      <td>3.009769</td>\n",
              "      <td>0.585600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>3.019887</td>\n",
              "      <td>0.585600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.061909</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.070799</td>\n",
              "      <td>0.587200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.082680</td>\n",
              "      <td>0.592000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.084583</td>\n",
              "      <td>0.590400</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.206353783607483).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 14:05, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.272300</td>\n",
              "      <td>1.212246</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.033900</td>\n",
              "      <td>1.128450</td>\n",
              "      <td>0.612800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.807800</td>\n",
              "      <td>1.156912</td>\n",
              "      <td>0.604800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.724500</td>\n",
              "      <td>1.307925</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.393600</td>\n",
              "      <td>1.393747</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.178800</td>\n",
              "      <td>1.664022</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.255400</td>\n",
              "      <td>1.889004</td>\n",
              "      <td>0.603200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.136700</td>\n",
              "      <td>2.065753</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.117900</td>\n",
              "      <td>2.459048</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.085900</td>\n",
              "      <td>2.544351</td>\n",
              "      <td>0.592000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.004500</td>\n",
              "      <td>2.655632</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.001800</td>\n",
              "      <td>2.830560</td>\n",
              "      <td>0.590400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.001100</td>\n",
              "      <td>2.925376</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.978410</td>\n",
              "      <td>0.584000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.002700</td>\n",
              "      <td>2.933573</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.054000</td>\n",
              "      <td>2.961990</td>\n",
              "      <td>0.603200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.021300</td>\n",
              "      <td>3.011384</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.034266</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.061054</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.048829</td>\n",
              "      <td>0.617600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.12844979763031).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:06, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.319900</td>\n",
              "      <td>1.205588</td>\n",
              "      <td>0.577600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.978900</td>\n",
              "      <td>1.116255</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.816300</td>\n",
              "      <td>1.130859</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.821600</td>\n",
              "      <td>1.280719</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.496800</td>\n",
              "      <td>1.309494</td>\n",
              "      <td>0.628800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.146600</td>\n",
              "      <td>1.554289</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.165500</td>\n",
              "      <td>1.672504</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.178600</td>\n",
              "      <td>1.918652</td>\n",
              "      <td>0.628800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.038300</td>\n",
              "      <td>2.086765</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.116000</td>\n",
              "      <td>2.294300</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.003500</td>\n",
              "      <td>2.416271</td>\n",
              "      <td>0.636800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.001500</td>\n",
              "      <td>2.489655</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.004700</td>\n",
              "      <td>2.526492</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>2.617174</td>\n",
              "      <td>0.627200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.019600</td>\n",
              "      <td>2.713682</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.661806</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.003200</td>\n",
              "      <td>2.688572</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.001600</td>\n",
              "      <td>2.701345</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.690373</td>\n",
              "      <td>0.654400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.695814</td>\n",
              "      <td>0.657600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.1162549257278442).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:14, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.139900</td>\n",
              "      <td>1.186527</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.909400</td>\n",
              "      <td>1.059664</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.685600</td>\n",
              "      <td>1.133391</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.607300</td>\n",
              "      <td>1.208220</td>\n",
              "      <td>0.635200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.316700</td>\n",
              "      <td>1.424255</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.182600</td>\n",
              "      <td>1.555377</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.190700</td>\n",
              "      <td>1.705964</td>\n",
              "      <td>0.624000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.098000</td>\n",
              "      <td>1.995148</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.007200</td>\n",
              "      <td>2.262979</td>\n",
              "      <td>0.617600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.013500</td>\n",
              "      <td>2.442546</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.002400</td>\n",
              "      <td>2.413244</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.002000</td>\n",
              "      <td>2.510874</td>\n",
              "      <td>0.640000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.002000</td>\n",
              "      <td>2.466660</td>\n",
              "      <td>0.657600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001000</td>\n",
              "      <td>2.643223</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.001400</td>\n",
              "      <td>2.766240</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.011600</td>\n",
              "      <td>2.755085</td>\n",
              "      <td>0.627200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.788893</td>\n",
              "      <td>0.636800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.808046</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.743892</td>\n",
              "      <td>0.644800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000400</td>\n",
              "      <td>2.778039</td>\n",
              "      <td>0.641600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.0596638917922974).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:02, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.268100</td>\n",
              "      <td>1.286879</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.961000</td>\n",
              "      <td>1.170761</td>\n",
              "      <td>0.590400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.691900</td>\n",
              "      <td>1.191083</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.642400</td>\n",
              "      <td>1.341316</td>\n",
              "      <td>0.622400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.312900</td>\n",
              "      <td>1.541527</td>\n",
              "      <td>0.624000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.284600</td>\n",
              "      <td>1.650567</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.248600</td>\n",
              "      <td>1.889238</td>\n",
              "      <td>0.606400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.123300</td>\n",
              "      <td>2.147868</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.161900</td>\n",
              "      <td>2.279927</td>\n",
              "      <td>0.612800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.043300</td>\n",
              "      <td>2.516886</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.015800</td>\n",
              "      <td>2.671912</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.081300</td>\n",
              "      <td>2.677503</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.013100</td>\n",
              "      <td>2.728430</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001700</td>\n",
              "      <td>2.875647</td>\n",
              "      <td>0.601600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.041100</td>\n",
              "      <td>2.786842</td>\n",
              "      <td>0.624000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.004300</td>\n",
              "      <td>3.022979</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.927167</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.001500</td>\n",
              "      <td>3.022665</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.997184</td>\n",
              "      <td>0.627200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.997494</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.1707608699798584).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:03, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.371100</td>\n",
              "      <td>1.301919</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.081300</td>\n",
              "      <td>1.196107</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.692400</td>\n",
              "      <td>1.225822</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.535300</td>\n",
              "      <td>1.311677</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.348000</td>\n",
              "      <td>1.458576</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.166300</td>\n",
              "      <td>1.601807</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.159400</td>\n",
              "      <td>1.825113</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.139200</td>\n",
              "      <td>2.099366</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.097400</td>\n",
              "      <td>2.326327</td>\n",
              "      <td>0.582400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.011300</td>\n",
              "      <td>2.486365</td>\n",
              "      <td>0.590400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.061800</td>\n",
              "      <td>2.600080</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.116500</td>\n",
              "      <td>2.727640</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.001800</td>\n",
              "      <td>2.828963</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>2.878491</td>\n",
              "      <td>0.593600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.032900</td>\n",
              "      <td>2.924203</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.021500</td>\n",
              "      <td>2.962484</td>\n",
              "      <td>0.611200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>3.000527</td>\n",
              "      <td>0.606400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.034651</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.018000</td>\n",
              "      <td>3.044721</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.053281</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.196107268333435).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3112' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3112/3660 22:06 < 03:53, 2.34 it/s, Epoch 17/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.292900</td>\n",
              "      <td>1.187130</td>\n",
              "      <td>0.603200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.051200</td>\n",
              "      <td>1.085045</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.789500</td>\n",
              "      <td>1.104119</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.691700</td>\n",
              "      <td>1.240097</td>\n",
              "      <td>0.606400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.346900</td>\n",
              "      <td>1.456409</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.258400</td>\n",
              "      <td>1.631499</td>\n",
              "      <td>0.635200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.291700</td>\n",
              "      <td>1.943097</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.063100</td>\n",
              "      <td>2.069870</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.150900</td>\n",
              "      <td>2.323310</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.007000</td>\n",
              "      <td>2.446229</td>\n",
              "      <td>0.612800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.005500</td>\n",
              "      <td>2.520805</td>\n",
              "      <td>0.622400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.001500</td>\n",
              "      <td>2.700134</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.000800</td>\n",
              "      <td>2.687340</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001300</td>\n",
              "      <td>2.867251</td>\n",
              "      <td>0.601600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.868524</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.934491</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.981354</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:08, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.292900</td>\n",
              "      <td>1.187130</td>\n",
              "      <td>0.603200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.051200</td>\n",
              "      <td>1.085045</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.789500</td>\n",
              "      <td>1.104119</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.691700</td>\n",
              "      <td>1.240097</td>\n",
              "      <td>0.606400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.346900</td>\n",
              "      <td>1.456409</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.258400</td>\n",
              "      <td>1.631499</td>\n",
              "      <td>0.635200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.291700</td>\n",
              "      <td>1.943097</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.063100</td>\n",
              "      <td>2.069870</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.150900</td>\n",
              "      <td>2.323310</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.007000</td>\n",
              "      <td>2.446229</td>\n",
              "      <td>0.612800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.005500</td>\n",
              "      <td>2.520805</td>\n",
              "      <td>0.622400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.001500</td>\n",
              "      <td>2.700134</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.000800</td>\n",
              "      <td>2.687340</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001300</td>\n",
              "      <td>2.867251</td>\n",
              "      <td>0.601600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.868524</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.934491</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.981354</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.003122</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.999850</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.998413</td>\n",
              "      <td>0.612800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.085044503211975).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        }
      ],
      "source": [
        "import csv\n",
        "import pickle\n",
        "import random\n",
        "import time\n",
        "\n",
        "from datasets import load_metric\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "from transformers import RobertaForSequenceClassification, RobertaTokenizerFast, Trainer, TrainingArguments\n",
        "\n",
        "def compute_metrics(eval_preds):\n",
        "    metric = load_metric(\"accuracy\")\n",
        "    logits, labels = eval_preds\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "    return metric.compute(predictions=predictions, references=labels)\n",
        "\n",
        "class PSCDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "def top_k_accuracy(top_k, predictions, labels):\n",
        "  assert len(predictions) == len(labels)\n",
        "  total = 0\n",
        "  correct = 0\n",
        "  for i in range(len(predictions)):\n",
        "    total += 1\n",
        "    prediction = []\n",
        "    for j, k in enumerate(predictions[i]):\n",
        "      prediction.append([j, k]) # k is the value\n",
        "    prediction.sort(key = lambda x: -x[1])\n",
        "    for j, _ in prediction[:top_k]:\n",
        "      if j == labels[i]:\n",
        "        correct += 1\n",
        "        break\n",
        "  return correct/total\n",
        "\n",
        "directory = \"./data_and_models/\"\n",
        "all_df = pd.read_csv(directory+\"target_corpus.csv\")\n",
        "epochs = 20\n",
        "seeds = [11, 12, 13, 14, 15]\n",
        "start = time.time()\n",
        "tokenizer = RobertaTokenizerFast.from_pretrained('roberta-base')\n",
        "\n",
        "tasks = {\n",
        "    \"44\": {\n",
        "        \"number_of_labels\": 42,\n",
        "         \"label_column\": 1,\n",
        "    },\n",
        "    \"8\": {\n",
        "        \"number_of_labels\": 8,\n",
        "        \"label_column\": 2,\n",
        "    }\n",
        "}\n",
        "\n",
        "def compute(task, mlength):\n",
        "  t1 =[] # t1: top 1 accuracy\n",
        "\n",
        "  for seed in seeds:\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    random.seed(seed)\n",
        "\n",
        "    from sklearn.model_selection import train_test_split\n",
        "\n",
        "    index = -1\n",
        "    classes = {}\n",
        "    texts = []\n",
        "    labels = []\n",
        "    lm_reverse_mapper = {}\n",
        "    with open(directory + \"target_corpus.csv\") as doc:\n",
        "      reader = csv.reader(doc)\n",
        "      next(reader)\n",
        "      for row in reader:\n",
        "        topic = row[tasks[task][\"label_column\"]]\n",
        "        if topic not in classes:\n",
        "          index += 1\n",
        "          classes[topic] = index\n",
        "          lm_reverse_mapper[index] = topic.capitalize()\n",
        "        labels.append(classes[topic])\n",
        "        texts.append(row[0])\n",
        "    print(\"# classes\", len(classes))\n",
        "    X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=625, random_state=seed)\n",
        "    X_train, X_dev, y_train, y_dev = train_test_split(X_train, y_train, test_size=625, random_state=seed)\n",
        "    print(len(X_train), len(X_dev), len(X_test))\n",
        "    print(\"# classes in train\", len(set(y_train)))\n",
        "    print(\"# classes in dev\", len(set(y_dev)))\n",
        "    print(\"# classes in test\", len(set(y_test)))\n",
        "\n",
        "    train_encodings = tokenizer(X_train, truncation=True, padding=True, max_length=mlength)\n",
        "    dev_encodings = tokenizer(X_dev, truncation=True, padding=True, max_length = mlength)\n",
        "    test_encodings = tokenizer(X_test, truncation=True, padding=True, max_length= mlength)\n",
        "\n",
        "\n",
        "    train_dataset = PSCDataset(train_encodings, y_train)\n",
        "    dev_dataset = PSCDataset(dev_encodings, y_dev)\n",
        "    test_dataset = PSCDataset(test_encodings, y_test)\n",
        "\n",
        "    training_args = TrainingArguments(\n",
        "        output_dir=\"./results\",          # output directory\n",
        "        num_train_epochs=epochs,         # total number of training epochs\n",
        "        per_device_train_batch_size=16,  # batch size per device during training\n",
        "        per_device_eval_batch_size=64,   # batch size for evaluation\n",
        "        warmup_steps=0,                  # number of warmup steps for learning rate scheduler\n",
        "        weight_decay=0.01,               # strength of weight decay\n",
        "        logging_dir='./logs',            # directory for storing logs\n",
        "        logging_steps=10,\n",
        "        learning_rate = 2e-5,\n",
        "        save_strategy= \"epoch\",\n",
        "        evaluation_strategy=\"epoch\",\n",
        "        load_best_model_at_end= True,\n",
        "        seed = seed, \n",
        "    )\n",
        "\n",
        "    def model_init():\n",
        "        return RobertaForSequenceClassification.from_pretrained(\"roberta-base\", num_labels=tasks[task][\"number_of_labels\"])\n",
        "    trainer = Trainer(\n",
        "        model_init=model_init,               # the instantiated 🤗 Transformers model to be trained\n",
        "        args=training_args,                  # training arguments, defined above\n",
        "        train_dataset=train_dataset,         # training dataset\n",
        "        eval_dataset=dev_dataset,            # evaluation dataset\n",
        "        compute_metrics=compute_metrics,     # compute_metrics\n",
        "        )\n",
        "\n",
        "    trainer.train()\n",
        "    predictions = trainer.predict(test_dataset)\n",
        "    preds = np.argmax(predictions.predictions, axis=-1)\n",
        "\n",
        "    t1.append(top_k_accuracy(1, predictions.predictions, test_dataset.labels))\n",
        "  return t1\n",
        "\n",
        "results = {}\n",
        "for task in tasks:\n",
        "  for mlength in [64, 256, 512]:\n",
        "    result = compute(task, mlength)\n",
        "    key = task + \"/\" + str(mlength)\n",
        "    results[key] = result\n",
        "\n",
        "np.save( directory + \"figure_1_appendix_results.npy\", results)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lLXYkZy1X7dh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "260d9cbf-328a-4b31-f78c-6c15b0fa7b25"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "44/64 \t (0.472, 0.014)\n",
            "44/256 \t (0.511, 0.015)\n",
            "44/512 \t (0.527, 0.013)\n",
            "8/64 \t (0.581, 0.014)\n",
            "8/256 \t (0.625, 0.007)\n",
            "8/512 \t (0.631, 0.015)\n"
          ]
        }
      ],
      "source": [
        "def process_result(result):\n",
        "  return round(np.mean(result), 3), round(np.std(result), 3)\n",
        "\n",
        "for key, result in results.items():\n",
        "  print(key, \"\\t\", process_result(result))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "spwbDwin_V0u",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e4af6e0-bd97-47a4-a21e-bf3ee040fab8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The program took 472.0 minutes in total.\n"
          ]
        }
      ],
      "source": [
        "end = time.time()\n",
        "print(f\"The program took {(end - start) // 60} minutes in total.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "urdHk_vbA9P-"
      },
      "outputs": [],
      "source": [
        "from google.colab import runtime\n",
        "runtime.unassign()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "005d0e44d16a4b8e95d207fce8f2fa9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "05517a93533449289149aa6b0e5fe672": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "09b71a436f2c495e97708494b6c59ad7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "12ea031fbddd4dd197ff322f89f60089": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "132e8a03c4954f70a1f787e5296fca37": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7048c2161e6a4b12b1f74aecc5847174",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8c988c814b3f48dea9de3eaa3255c91f",
            "value": 456318
          }
        },
        "171fca9859d14cca9f314b1675e7e54c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "17cafcc34f374c1eb0f1f95f609549b5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "187c716e512548a697074e5bcce4fc18": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1919fae16694484b94f4272c52b5d97b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d099a73be5f46b883c1f9d463b7a424": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1dd8e0c5438d47e69beb4ba0a47d6cab": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_21f3f43494a0413fbe736fa41c7dfa51",
              "IPY_MODEL_132e8a03c4954f70a1f787e5296fca37",
              "IPY_MODEL_3b43d94ef3c64a25915fee2643d9c8c6"
            ],
            "layout": "IPY_MODEL_45b2aa984e904e5aa4297347177c7b82"
          }
        },
        "21f3343889134db39bb10bbdd26ff6a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "21f3f43494a0413fbe736fa41c7dfa51": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b053ad4929094530aee19358b3e91669",
            "placeholder": "​",
            "style": "IPY_MODEL_1d099a73be5f46b883c1f9d463b7a424",
            "value": "Downloading: 100%"
          }
        },
        "2520b10f52084ceca5d66dbfcb00d976": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "298577d50d844021b028a243adc57cde": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bcb9a01f77ae4cd6b0520f4e0648df73",
            "placeholder": "​",
            "style": "IPY_MODEL_09b71a436f2c495e97708494b6c59ad7",
            "value": " 4.21k/? [00:00&lt;00:00, 163kB/s]"
          }
        },
        "2b6b3ee87c7a4fb492d64d8da1e0845c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f278b8cd217d408c8b17d476eee0e73e",
            "placeholder": "​",
            "style": "IPY_MODEL_005d0e44d16a4b8e95d207fce8f2fa9b",
            "value": "Downloading: 100%"
          }
        },
        "2fa8f82a5c8c413e8d8ea47f48dd47ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3620c805f5d741cab73c251c9447e8c2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "391c778ce2c64f1e92445c1de8084bee": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3b3d68390f134798a2a9bbc73a1081e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e274353afc5f42fa92e4eaf3bbda876a",
            "placeholder": "​",
            "style": "IPY_MODEL_2fa8f82a5c8c413e8d8ea47f48dd47ba",
            "value": " 481/481 [00:00&lt;00:00, 19.4kB/s]"
          }
        },
        "3b43d94ef3c64a25915fee2643d9c8c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1919fae16694484b94f4272c52b5d97b",
            "placeholder": "​",
            "style": "IPY_MODEL_87c405bbac3f48d8b799816335199f12",
            "value": " 456k/456k [00:00&lt;00:00, 1.79MB/s]"
          }
        },
        "43a1275ed3a846398c0d2e5dcdb54048": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "45651d7475c34e5ea03b89fe35e644ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4910017bb94482c907310eb6e133ffd",
            "placeholder": "​",
            "style": "IPY_MODEL_50cd3b6daa0c4a4588b02740fa4fc463",
            "value": "Downloading: 100%"
          }
        },
        "45b2aa984e904e5aa4297347177c7b82": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50cd3b6daa0c4a4588b02740fa4fc463": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "52bed226b35641bea40884bc01f3ae2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3620c805f5d741cab73c251c9447e8c2",
            "max": 1652,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b6f89438b3c946f8bceeda59d5208f4d",
            "value": 1652
          }
        },
        "543677234ab04574b17edbeb293f09aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f8d5b4c2d20d4e2dbaad7dafdd391bc2",
            "placeholder": "​",
            "style": "IPY_MODEL_171fca9859d14cca9f314b1675e7e54c",
            "value": " 501M/501M [00:08&lt;00:00, 56.5MB/s]"
          }
        },
        "58ff4e5eec7d43b8b362c60f8535879f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60e808b79ad448028dcf9429208c7018": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "612d9949943e428ea7b1b263691261a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b3a09d45a792485c8db365c53ab5dd8d",
            "placeholder": "​",
            "style": "IPY_MODEL_2520b10f52084ceca5d66dbfcb00d976",
            "value": " 899k/899k [00:00&lt;00:00, 1.59MB/s]"
          }
        },
        "6fc774ec1962478aa9e2490d6e597a2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_391c778ce2c64f1e92445c1de8084bee",
            "placeholder": "​",
            "style": "IPY_MODEL_ecf3b77d0099414d93bb05877dc1bc8a",
            "value": "Downloading: 100%"
          }
        },
        "7048c2161e6a4b12b1f74aecc5847174": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7af66c5f2e2948138942375ace325b75": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b091adad6c448f0a465992d10a25781": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f259ab6c6f84b388ab914e3fb3f3804": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ff32ea967064f8d88de00e9b23f1384": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "81e4a44d5ee24f27b1e24ffa6aefb787": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84ffa11876364d5791331c0a8b905397": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85d636083d354d398755f364ae08811e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "861a0c463424460888fa7dceba3b67d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a013e26d3b82443681f197fba9d212c1",
              "IPY_MODEL_52bed226b35641bea40884bc01f3ae2b",
              "IPY_MODEL_298577d50d844021b028a243adc57cde"
            ],
            "layout": "IPY_MODEL_e0bc4793a7b04c3ab660a37401fa1cce"
          }
        },
        "86c2471c24e24765a3a102f3d128d11c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "87c405bbac3f48d8b799816335199f12": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "882495f4154e4cb7ab41da052bcc56cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12ea031fbddd4dd197ff322f89f60089",
            "max": 898823,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fe54405e547a4ebe80f663d294df33e9",
            "value": 898823
          }
        },
        "89cb305328b14ca2b6b485161fe0d295": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_58ff4e5eec7d43b8b362c60f8535879f",
            "max": 481,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7ff32ea967064f8d88de00e9b23f1384",
            "value": 481
          }
        },
        "8c988c814b3f48dea9de3eaa3255c91f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "93ceb2df65bf437d970adc95df8b620f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_60e808b79ad448028dcf9429208c7018",
            "max": 1355863,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_187c716e512548a697074e5bcce4fc18",
            "value": 1355863
          }
        },
        "a013e26d3b82443681f197fba9d212c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81e4a44d5ee24f27b1e24ffa6aefb787",
            "placeholder": "​",
            "style": "IPY_MODEL_7b091adad6c448f0a465992d10a25781",
            "value": "Downloading builder script: "
          }
        },
        "a4e6dddfe191432b961b506564cccddd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_85d636083d354d398755f364ae08811e",
            "placeholder": "​",
            "style": "IPY_MODEL_43a1275ed3a846398c0d2e5dcdb54048",
            "value": "Downloading: 100%"
          }
        },
        "b053ad4929094530aee19358b3e91669": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b3a09d45a792485c8db365c53ab5dd8d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6f89438b3c946f8bceeda59d5208f4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bb210bf5173e4090a702fc3aa5995e46": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a4e6dddfe191432b961b506564cccddd",
              "IPY_MODEL_93ceb2df65bf437d970adc95df8b620f",
              "IPY_MODEL_d61cdb598a94451486c9adfeffaeefd5"
            ],
            "layout": "IPY_MODEL_7f259ab6c6f84b388ab914e3fb3f3804"
          }
        },
        "bcb9a01f77ae4cd6b0520f4e0648df73": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfeb8cae1ada47c1aed45b4770cc80ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2b6b3ee87c7a4fb492d64d8da1e0845c",
              "IPY_MODEL_882495f4154e4cb7ab41da052bcc56cd",
              "IPY_MODEL_612d9949943e428ea7b1b263691261a4"
            ],
            "layout": "IPY_MODEL_86c2471c24e24765a3a102f3d128d11c"
          }
        },
        "d61cdb598a94451486c9adfeffaeefd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fcb499e5829c4c8da7985081ac48b9a3",
            "placeholder": "​",
            "style": "IPY_MODEL_05517a93533449289149aa6b0e5fe672",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 2.29MB/s]"
          }
        },
        "dffcb57ad5884a46a9020a3cbf6ce5e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_17cafcc34f374c1eb0f1f95f609549b5",
            "max": 501200538,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_21f3343889134db39bb10bbdd26ff6a6",
            "value": 501200538
          }
        },
        "e0bc4793a7b04c3ab660a37401fa1cce": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e274353afc5f42fa92e4eaf3bbda876a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4910017bb94482c907310eb6e133ffd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4d1f51a95ba4e43874cc30c677c3387": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_45651d7475c34e5ea03b89fe35e644ab",
              "IPY_MODEL_89cb305328b14ca2b6b485161fe0d295",
              "IPY_MODEL_3b3d68390f134798a2a9bbc73a1081e7"
            ],
            "layout": "IPY_MODEL_7af66c5f2e2948138942375ace325b75"
          }
        },
        "ebc1ad964f8f47d2808a2c92556af0a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6fc774ec1962478aa9e2490d6e597a2a",
              "IPY_MODEL_dffcb57ad5884a46a9020a3cbf6ce5e5",
              "IPY_MODEL_543677234ab04574b17edbeb293f09aa"
            ],
            "layout": "IPY_MODEL_84ffa11876364d5791331c0a8b905397"
          }
        },
        "ecf3b77d0099414d93bb05877dc1bc8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f278b8cd217d408c8b17d476eee0e73e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f8d5b4c2d20d4e2dbaad7dafdd391bc2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fcb499e5829c4c8da7985081ac48b9a3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe54405e547a4ebe80f663d294df33e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}